{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HW6_Mnist_By_Ztt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting input_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "#导入数据\n",
    "data_dir = 'input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=tf.placeholder(tf.float32,[None,784])\n",
    "\n",
    "h1_num =600\n",
    "h2_num=400\n",
    "h3_num=200\n",
    "h4_num=100\n",
    "\n",
    "W1 = tf.Variable(tf.truncated_normal([784,h1_num],stddev=0.1))#前面用初始为0的时候各种坑.....\n",
    "b1 = tf.Variable(tf.zeros([h1_num])+0.1)\n",
    "h1=tf.nn.relu(tf.matmul(X, W1) + b1)#这个用elu轻轻松松 准确率就上去了...\n",
    "\n",
    "W2=tf.Variable(tf.truncated_normal([h1_num,h2_num],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([h2_num])+0.1)\n",
    "h2=tf.nn.relu(tf.matmul(h1, W2) + b2)\n",
    "\n",
    "W3=tf.Variable(tf.truncated_normal([h2_num,h3_num],stddev=0.1))\n",
    "b3= tf.Variable(tf.zeros([h3_num]))\n",
    "h3=tf.nn.relu(tf.matmul(h2, W3) + b3)\n",
    "\n",
    "W4=tf.Variable(tf.truncated_normal([h3_num,h4_num],stddev=0.1))\n",
    "b4= tf.Variable(tf.zeros([h4_num]))\n",
    "h4=tf.nn.relu(tf.matmul(h3, W4) + b4)\n",
    "\n",
    "W5=tf.Variable(tf.truncated_normal([h4_num,10],stddev=0.1))\n",
    "b5=tf.Variable(tf.zeros([10]))\n",
    "\n",
    "y=tf.matmul(h4, W5) + b5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#创建损失\n",
    "#添加l2正则\n",
    "gamma=0.001\n",
    "regular=(tf.contrib.layers.l2_regularizer(gamma)(W1)+\n",
    "         tf.contrib.layers.l2_regularizer(gamma)(W2)+\n",
    "         tf.contrib.layers.l2_regularizer(gamma)(W3)+\n",
    "         tf.contrib.layers.l2_regularizer(gamma)(W4)+\n",
    "         tf.contrib.layers.l2_regularizer(gamma)(W5))\n",
    "lost=tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)+regular\n",
    "cross_entropy = tf.reduce_mean(lost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#这里是网上参考的一种设置衰减的增长率的方法\n",
    "#learn_rate=tf.train.exponential_decay(learning_rate=0.1,decay_rate=0.96,decay_steps=100,global_step=3000,staircase=False)\n",
    "#train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy)\n",
    "\n",
    "learn_rate=tf.Variable(0.001, dtype=tf.float32)\n",
    "train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy)\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()#返回一个用来初始化 计算图中 所有global variable的 op。 \n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0007694497527671316"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0.01*(0.95**50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第0次迭代的accuracy--0.9218000173568726 leanr_rate-- 0.001\n",
      "第100次迭代的accuracy--0.9448000192642212 leanr_rate-- 0.00095\n",
      "第200次迭代的accuracy--0.9562000036239624 leanr_rate-- 0.0009025\n",
      "第300次迭代的accuracy--0.9531000256538391 leanr_rate-- 0.000857375\n",
      "第400次迭代的accuracy--0.9624999761581421 leanr_rate-- 0.00081450626\n",
      "第500次迭代的accuracy--0.9610000252723694 leanr_rate-- 0.0007737809\n",
      "第600次迭代的accuracy--0.9607999920845032 leanr_rate-- 0.0007350919\n",
      "第700次迭代的accuracy--0.9707000255584717 leanr_rate-- 0.0006983373\n",
      "第800次迭代的accuracy--0.9682000279426575 leanr_rate-- 0.0006634204\n",
      "第900次迭代的accuracy--0.9675999879837036 leanr_rate-- 0.0006302494\n",
      "第1000次迭代的accuracy--0.9678000211715698 leanr_rate-- 0.0005987369\n",
      "第1100次迭代的accuracy--0.9726999998092651 leanr_rate-- 0.0005688001\n",
      "第1200次迭代的accuracy--0.9739000201225281 leanr_rate-- 0.0005403601\n",
      "第1300次迭代的accuracy--0.9722999930381775 leanr_rate-- 0.0005133421\n",
      "第1400次迭代的accuracy--0.9746999740600586 leanr_rate-- 0.000487675\n",
      "第1500次迭代的accuracy--0.9740999937057495 leanr_rate-- 0.00046329122\n",
      "第1600次迭代的accuracy--0.9772999882698059 leanr_rate-- 0.00044012666\n",
      "第1700次迭代的accuracy--0.9783999919891357 leanr_rate-- 0.00041812033\n",
      "第1800次迭代的accuracy--0.9739000201225281 leanr_rate-- 0.00039721432\n",
      "第1900次迭代的accuracy--0.978600025177002 leanr_rate-- 0.0003773536\n",
      "第2000次迭代的accuracy--0.9799000024795532 leanr_rate-- 0.00035848594\n",
      "第2100次迭代的accuracy--0.9769999980926514 leanr_rate-- 0.00034056162\n",
      "第2200次迭代的accuracy--0.9793999791145325 leanr_rate-- 0.00032353355\n",
      "第2300次迭代的accuracy--0.9783999919891357 leanr_rate-- 0.00030735688\n",
      "第2400次迭代的accuracy--0.9781000018119812 leanr_rate-- 0.000291989\n",
      "第2500次迭代的accuracy--0.9797999858856201 leanr_rate-- 0.00027738957\n",
      "第2600次迭代的accuracy--0.9789000153541565 leanr_rate-- 0.0002635201\n",
      "第2700次迭代的accuracy--0.9801999926567078 leanr_rate-- 0.00025034408\n",
      "第2800次迭代的accuracy--0.9815000295639038 leanr_rate-- 0.00023782688\n",
      "第2900次迭代的accuracy--0.9811999797821045 leanr_rate-- 0.00022593554\n",
      "第3000次迭代的accuracy--0.9815000295639038 leanr_rate-- 0.00021463877\n",
      "第3100次迭代的accuracy--0.9817000031471252 leanr_rate-- 0.00020390682\n",
      "第3200次迭代的accuracy--0.98089998960495 leanr_rate-- 0.00019371149\n",
      "第3300次迭代的accuracy--0.9815999865531921 leanr_rate-- 0.0001840259\n",
      "第3400次迭代的accuracy--0.9818000197410583 leanr_rate-- 0.00017482461\n",
      "第3500次迭代的accuracy--0.9800000190734863 leanr_rate-- 0.00016608338\n",
      "第3600次迭代的accuracy--0.9817000031471252 leanr_rate-- 0.00015777921\n",
      "第3700次迭代的accuracy--0.9800000190734863 leanr_rate-- 0.00014989026\n",
      "第3800次迭代的accuracy--0.9811999797821045 leanr_rate-- 0.00014239574\n",
      "第3900次迭代的accuracy--0.9812999963760376 leanr_rate-- 0.00013527596\n",
      "第4000次迭代的accuracy--0.9832000136375427 leanr_rate-- 0.00012851215\n",
      "第4100次迭代的accuracy--0.9814000129699707 leanr_rate-- 0.00012208655\n",
      "第4200次迭代的accuracy--0.980400025844574 leanr_rate-- 0.00011598222\n",
      "第4300次迭代的accuracy--0.9818000197410583 leanr_rate-- 0.00011018311\n",
      "第4400次迭代的accuracy--0.9821000099182129 leanr_rate-- 0.000104673956\n",
      "第4500次迭代的accuracy--0.9836000204086304 leanr_rate-- 9.944026e-05\n",
      "第4600次迭代的accuracy--0.9815999865531921 leanr_rate-- 9.446825e-05\n",
      "第4700次迭代的accuracy--0.9814000129699707 leanr_rate-- 8.974483e-05\n",
      "第4800次迭代的accuracy--0.9825999736785889 leanr_rate-- 8.525759e-05\n",
      "第4900次迭代的accuracy--0.9821000099182129 leanr_rate-- 8.099471e-05\n"
     ]
    }
   ],
   "source": [
    "# 训练\n",
    "for n in range(50):\n",
    "    sess.run(tf.assign(learn_rate, 0.001 * (0.95 ** n)))#将learn_rate进行变化\n",
    "    for i in range(100):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        sess.run(train_step, feed_dict={X: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(\"第{}次迭代的accuracy--{}\".format(n*100,sess.run(accuracy, feed_dict={X: mnist.test.images,y_: mnist.test.labels})),\n",
    "                 \"leanr_rate--\",sess.run(learn_rate))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "走了好多坑，终于达到0.98了，前面一直卡在0.979好久，怎么都上不了0.98，我发现想要上0.98主要的问题还是优化器的选择上面，还有激活函数的选择，用elu是上不去的，不是说elu是relu的进阶版本么，不懂....优化器用普通梯度下降无论怎么样都上不了0.98...换了这个Adam算法的优化器后就很轻松了...虽然还不知道这个的算法是怎样的...另外一个为什么用l2正则之后，结果不但没多大提升，反而有所下降，然后怎么判断要用多少个隐层，以及隐层内神经元的数量呢？我调这个感觉就是一个玄学。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
